#!/bin/bash

# EKS Persistent Configuration Detection Script
# This script analyzes EKS cluster configurations to identify custom settings
# that need to be preserved during add-on upgrades

# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
PURPLE='\033[0;35m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color

# Print functions
print_info() {
    echo -e "${BLUE}[INFO]${NC} $1"
}

print_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

print_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

print_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

print_custom() {
    echo -e "${PURPLE}[CUSTOM]${NC} $1"
}

print_default() {
    echo -e "${CYAN}[DEFAULT]${NC} $1"
}

# Function to show usage
show_usage() {
    cat << EOF
Usage: $0 -c CONFIG_DIR [OPTIONS]

Required:
  -c, --config      EKS configuration directory (from collect-eks-nodegroup-config.sh)

Optional:
  -o, --output      Output directory for persistence recommendations (default: ./persistence-analysis-TIMESTAMP)
  -f, --format      Output format: text|json|yaml (default: text)
  -v, --verbose     Verbose output showing all analysis
  -s, --severity    Minimum severity level: low|medium|high|critical (default: medium)
  -a, --addon       Analyze only specific addon: vpc-cni|coredns|kube-proxy|aws-ebs-csi-driver|aws-efs-csi-driver
  -h, --help        Show this help message

Examples:
  $0 -c ./eks-nodegroup-config-20250816-235819
  $0 -c ./eks-config -o ./persistence-report -f yaml -v
  $0 -c ./eks-config -s high
  $0 -c ./eks-config -a vpc-cni -v
  $0 -c ./eks-config -a coredns -f json

Description:
This script analyzes EKS cluster configurations to identify:
- Custom add-on configurations that differ from defaults
- Modified environment variables in DaemonSets/Deployments
- Custom ConfigMap settings
- Non-standard resource limits and requests
- Custom IAM policies and roles
- Modified security group rules
- Custom CRD configurations

The script generates recommendations for preserving these configurations during upgrades.

EOF
}

# Parse command line arguments
CONFIG_DIR=""
OUTPUT_DIR=""
FORMAT="text"
VERBOSE=false
SEVERITY="medium"
SINGLE_ADDON=""

while [[ $# -gt 0 ]]; do
    case $1 in
        -c|--config)
            CONFIG_DIR="$2"
            shift 2
            ;;
        -o|--output)
            OUTPUT_DIR="$2"
            shift 2
            ;;
        -f|--format)
            FORMAT="$2"
            shift 2
            ;;
        -v|--verbose)
            VERBOSE=true
            shift
            ;;
        -s|--severity)
            SEVERITY="$2"
            shift 2
            ;;
        -a|--addon)
            SINGLE_ADDON="$2"
            shift 2
            ;;
        -h|--help)
            show_usage
            exit 0
            ;;
        *)
            echo "Unknown option $1"
            show_usage
            exit 1
            ;;
    esac
done

# Validate required parameters
if [[ -z "$CONFIG_DIR" ]]; then
    print_error "Configuration directory is required"
    show_usage
    exit 1
fi

# Validate directory exists
if [[ ! -d "$CONFIG_DIR" ]]; then
    print_error "Configuration directory does not exist: $CONFIG_DIR"
    exit 1
fi

# Validate addon parameter if provided
if [[ -n "$SINGLE_ADDON" ]]; then
    case "$SINGLE_ADDON" in
        vpc-cni|coredns|kube-proxy|aws-ebs-csi-driver|aws-efs-csi-driver|aws-mountpoint-s3-csi-driver)
            print_info "Single addon mode: analyzing only $SINGLE_ADDON"
            ;;
        *)
            print_error "Invalid addon name: $SINGLE_ADDON"
            print_error "Valid addons: vpc-cni, coredns, kube-proxy, aws-ebs-csi-driver, aws-efs-csi-driver, aws-mountpoint-s3-csi-driver"
            exit 1
            ;;
    esac
fi

# Set default output directory
if [[ -z "$OUTPUT_DIR" ]]; then
    OUTPUT_DIR="./persistence-analysis-$(date +%Y%m%d-%H%M%S)"
fi

# Create output directory
mkdir -p "$OUTPUT_DIR"

print_info "Starting persistent configuration analysis"
print_info "Config Directory: $CONFIG_DIR"
print_info "Output Directory: $OUTPUT_DIR"
print_info "Format: $FORMAT"
print_info "Severity Filter: $SEVERITY"
if [[ -n "$SINGLE_ADDON" ]]; then
    print_info "Single Addon Mode: $SINGLE_ADDON"
fi

# Initialize results
RESULTS_FILE="$OUTPUT_DIR/persistence-analysis.json"
echo '{"timestamp":"'$(date -Iseconds)'","config_dir":"'$CONFIG_DIR'","analysis":{}}' > "$RESULTS_FILE"

# Function to add analysis result
add_analysis_result() {
    local category="$1"
    local component="$2"
    local severity="$3"
    local finding="$4"
    local recommendation="$5"
    local config_snippet="$6"
    
    # Create temporary file with updated JSON
    jq --arg cat "$category" --arg comp "$component" --arg sev "$severity" \
       --arg find "$finding" --arg rec "$recommendation" --arg snippet "$config_snippet" \
       '.analysis[$cat] += [{
         component: $comp,
         severity: $sev,
         finding: $find,
         recommendation: $rec,
         config_snippet: $snippet
       }]' "$RESULTS_FILE" > "$RESULTS_FILE.tmp" && mv "$RESULTS_FILE.tmp" "$RESULTS_FILE"
}

# Function to check severity level
should_report() {
    local item_severity="$1"
    
    case "$SEVERITY" in
        "low")
            return 0
            ;;
        "medium")
            [[ "$item_severity" =~ ^(medium|high|critical)$ ]]
            ;;
        "high")
            [[ "$item_severity" =~ ^(high|critical)$ ]]
            ;;
        "critical")
            [[ "$item_severity" == "critical" ]]
            ;;
        *)
            return 0
            ;;
    esac
}

# Known default configurations for comparison
get_vpc_cni_default() {
    case "$1" in
        "ENABLE_PREFIX_DELEGATION") echo "false" ;;
        "ENABLE_POD_ENI") echo "false" ;;
        "ENABLE_IPv6") echo "false" ;;
        "ENABLE_IPv4") echo "true" ;;
        "WARM_PREFIX_TARGET") echo "1" ;;
        "MINIMUM_IP_TARGET") echo "3" ;;
        "WARM_IP_TARGET") echo "1" ;;
        "AWS_VPC_K8S_CNI_EXTERNALSNAT") echo "false" ;;
        "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG") echo "false" ;;
        "ENABLE_SUBNET_DISCOVERY") echo "true" ;;
        "DISABLE_NETWORK_RESOURCE_PROVISIONING") echo "false" ;;
        *) echo "" ;;
    esac
}

get_coredns_default() {
    case "$1" in
        "replicas") echo "2" ;;
        "resources_limits_memory") echo "170Mi" ;;
        "resources_requests_memory") echo "70Mi" ;;
        "resources_requests_cpu") echo "100m" ;;
        *) echo "" ;;
    esac
}

get_kube_proxy_default() {
    case "$1" in
        "resources_requests_cpu") echo "100m" ;;
        "resources_requests_memory") echo "64Mi" ;;
        *) echo "" ;;
    esac
}

# Function to analyze VPC-CNI configuration
analyze_vpc_cni() {
    print_info "=== Analyzing VPC-CNI Configuration ==="
    
    local vpc_cni_ds="$CONFIG_DIR/addon-k8s-resources/vpc-cni-daemonset.yaml"
    local vpc_cni_cm="$CONFIG_DIR/addon-k8s-resources/vpc-cni-configmap.yaml"
    local vpc_cni_addon="$CONFIG_DIR/addons/vpc-cni.json"
    
    # Check EKS managed add-on configuration
    if [[ -f "$vpc_cni_addon" ]]; then
        local custom_config=$(jq -r '.addon.configurationValues // empty' "$vpc_cni_addon" 2>/dev/null)
        if [[ -n "$custom_config" && "$custom_config" != "null" ]]; then
            print_custom "VPC-CNI has custom EKS managed configuration"
            add_analysis_result "eks_addons" "vpc-cni" "high" \
                "Custom EKS managed add-on configuration detected" \
                "Preserve using: aws eks update-addon --configuration-values '$custom_config'" \
                "$custom_config"
        elif [[ "$VERBOSE" == "true" ]]; then
            print_default "VPC-CNI using default EKS managed configuration"
        fi
    fi
    
    # Check DaemonSet environment variables
    if [[ -f "$vpc_cni_ds" ]]; then
        print_info "Analyzing VPC-CNI DaemonSet environment variables..."
        
        # Extract environment variables
        if command -v yq >/dev/null 2>&1; then
            local env_vars=$(yq eval '.spec.template.spec.containers[0].env[] | .name + "=" + .value' "$vpc_cni_ds" 2>/dev/null)
        else
            # Fallback to grep if yq not available - extract env vars from YAML
            local env_vars=$(grep -A 1 "- name:" "$vpc_cni_ds" | grep -E "(name|value):" | sed 'N;s/.*name: \(.*\)\n.*value: \(.*\)/\1=\2/' | grep "=")
        fi
        
        while IFS= read -r env_line; do
            if [[ -n "$env_line" ]]; then
                local env_name=$(echo "$env_line" | cut -d'=' -f1)
                local env_value=$(echo "$env_line" | cut -d'=' -f2-)
                
                # Check against known defaults
                local default_value=$(get_vpc_cni_default "$env_name")
                if [[ -n "$default_value" ]]; then
                    if [[ "$env_value" != "$default_value" ]]; then
                        print_custom "VPC-CNI custom setting: $env_name=$env_value (default: $default_value)"
                        
                        local severity="medium"
                        local recommendation="kubectl patch daemonset aws-node -n kube-system -p='{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"aws-node\",\"env\":[{\"name\":\"$env_name\",\"value\":\"$env_value\"}]}]}}}}'"
                        
                        # Determine severity based on setting importance
                        case "$env_name" in
                            "ENABLE_PREFIX_DELEGATION"|"ENABLE_POD_ENI"|"AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG")
                                severity="high"
                                ;;
                            "WARM_PREFIX_TARGET"|"MINIMUM_IP_TARGET"|"WARM_IP_TARGET")
                                severity="medium"
                                ;;
                            *)
                                severity="low"
                                ;;
                        esac
                        
                        if should_report "$severity"; then
                            add_analysis_result "k8s_resources" "vpc-cni-daemonset" "$severity" \
                                "Custom environment variable: $env_name=$env_value" \
                                "$recommendation" \
                                "$env_name=$env_value"
                        fi
                    elif [[ "$VERBOSE" == "true" ]]; then
                        print_default "VPC-CNI default setting: $env_name=$env_value"
                    fi
                else
                    # Unknown environment variable (potentially custom)
                    print_custom "VPC-CNI unknown/custom setting: $env_name=$env_value"
                    if should_report "medium"; then
                        add_analysis_result "k8s_resources" "vpc-cni-daemonset" "medium" \
                            "Unknown/custom environment variable: $env_name=$env_value" \
                            "Verify if this is a custom setting that needs preservation" \
                            "$env_name=$env_value"
                    fi
                fi
            fi
        done <<< "$env_vars"
        
        # Check resource limits and requests
        if command -v yq >/dev/null 2>&1; then
            local cpu_limit=$(yq eval '.spec.template.spec.containers[0].resources.limits.cpu // "none"' "$vpc_cni_ds" 2>/dev/null)
            local memory_limit=$(yq eval '.spec.template.spec.containers[0].resources.limits.memory // "none"' "$vpc_cni_ds" 2>/dev/null)
            local cpu_request=$(yq eval '.spec.template.spec.containers[0].resources.requests.cpu // "none"' "$vpc_cni_ds" 2>/dev/null)
            local memory_request=$(yq eval '.spec.template.spec.containers[0].resources.requests.memory // "none"' "$vpc_cni_ds" 2>/dev/null)
            
            if [[ "$cpu_limit" != "none" || "$memory_limit" != "none" || "$cpu_request" != "none" || "$memory_request" != "none" ]]; then
                print_custom "VPC-CNI has custom resource limits/requests"
                if should_report "medium"; then
                    add_analysis_result "k8s_resources" "vpc-cni-daemonset" "medium" \
                        "Custom resource limits/requests: CPU limit=$cpu_limit, Memory limit=$memory_limit, CPU request=$cpu_request, Memory request=$memory_request" \
                        "Preserve resource settings in DaemonSet configuration" \
                        "resources: {limits: {cpu: $cpu_limit, memory: $memory_limit}, requests: {cpu: $cpu_request, memory: $memory_request}}"
                fi
            fi
        fi
    fi
    
    # Check ConfigMap
    if [[ -f "$vpc_cni_cm" ]]; then
        print_info "Analyzing VPC-CNI ConfigMap..."
        
        if command -v yq >/dev/null 2>&1; then
            local cm_data=$(yq eval '.data // {}' "$vpc_cni_cm" 2>/dev/null)
            if [[ "$cm_data" != "{}" && "$cm_data" != "null" ]]; then
                print_custom "VPC-CNI has custom ConfigMap data"
                if should_report "medium"; then
                    add_analysis_result "k8s_resources" "vpc-cni-configmap" "medium" \
                        "Custom ConfigMap configuration detected" \
                        "Preserve ConfigMap data during upgrades" \
                        "$cm_data"
                fi
            fi
        fi
    fi
}

# Function to analyze CoreDNS configuration
analyze_coredns() {
    print_info "=== Analyzing CoreDNS Configuration ==="
    
    local coredns_deploy="$CONFIG_DIR/addon-k8s-resources/coredns-deployment.yaml"
    local coredns_cm="$CONFIG_DIR/addon-k8s-resources/coredns-configmap.yaml"
    local coredns_addon="$CONFIG_DIR/addons/coredns.json"
    
    # Check EKS managed add-on configuration
    if [[ -f "$coredns_addon" ]]; then
        local custom_config=$(jq -r '.addon.configurationValues // empty' "$coredns_addon" 2>/dev/null)
        if [[ -n "$custom_config" && "$custom_config" != "null" ]]; then
            print_custom "CoreDNS has custom EKS managed configuration"
            add_analysis_result "eks_addons" "coredns" "high" \
                "Custom EKS managed add-on configuration detected" \
                "Preserve using: aws eks update-addon --configuration-values '$custom_config'" \
                "$custom_config"
        fi
    fi
    
    # Check Deployment configuration
    if [[ -f "$coredns_deploy" ]] && command -v yq >/dev/null 2>&1; then
        print_info "Analyzing CoreDNS Deployment configuration..."
        
        # Check replica count
        local replicas=$(yq eval '.spec.replicas // 2' "$coredns_deploy" 2>/dev/null)
        if [[ "$replicas" != "2" ]]; then
            print_custom "CoreDNS custom replica count: $replicas"
            if should_report "medium"; then
                add_analysis_result "k8s_resources" "coredns-deployment" "medium" \
                    "Custom replica count: $replicas (default: 2)" \
                    "kubectl scale deployment coredns -n kube-system --replicas=$replicas" \
                    "replicas: $replicas"
            fi
        fi
        
        # Check resource limits
        local memory_limit=$(yq eval '.spec.template.spec.containers[0].resources.limits.memory // "170Mi"' "$coredns_deploy" 2>/dev/null)
        local memory_request=$(yq eval '.spec.template.spec.containers[0].resources.requests.memory // "70Mi"' "$coredns_deploy" 2>/dev/null)
        local cpu_request=$(yq eval '.spec.template.spec.containers[0].resources.requests.cpu // "100m"' "$coredns_deploy" 2>/dev/null)
        
        if [[ "$memory_limit" != "170Mi" || "$memory_request" != "70Mi" || "$cpu_request" != "100m" ]]; then
            print_custom "CoreDNS custom resource settings"
            if should_report "medium"; then
                add_analysis_result "k8s_resources" "coredns-deployment" "medium" \
                    "Custom resource settings: memory limit=$memory_limit, memory request=$memory_request, cpu request=$cpu_request" \
                    "Preserve resource settings in Deployment configuration" \
                    "resources: {limits: {memory: $memory_limit}, requests: {memory: $memory_request, cpu: $cpu_request}}"
            fi
        fi
    fi
    
    # Check ConfigMap (Corefile)
    if [[ -f "$coredns_cm" ]] && command -v yq >/dev/null 2>&1; then
        print_info "Analyzing CoreDNS ConfigMap (Corefile)..."
        
        local corefile=$(yq eval '.data.Corefile // empty' "$coredns_cm" 2>/dev/null)
        if [[ -n "$corefile" ]]; then
            # Check for common customizations
            if echo "$corefile" | grep -q "forward.*8.8.8.8\|forward.*1.1.1.1"; then
                print_custom "CoreDNS custom DNS forwarders detected"
                if should_report "high"; then
                    add_analysis_result "k8s_resources" "coredns-configmap" "high" \
                        "Custom DNS forwarders in Corefile" \
                        "Preserve Corefile configuration during upgrades" \
                        "$(echo "$corefile" | grep "forward")"
                fi
            fi
            
            if echo "$corefile" | grep -q "cache.*[0-9]"; then
                local cache_setting=$(echo "$corefile" | grep -o "cache [0-9]*")
                print_custom "CoreDNS custom cache setting: $cache_setting"
                if should_report "medium"; then
                    add_analysis_result "k8s_resources" "coredns-configmap" "medium" \
                        "Custom cache setting: $cache_setting" \
                        "Preserve Corefile cache configuration" \
                        "$cache_setting"
                fi
            fi
            
            if echo "$corefile" | grep -qE "log|debug"; then
                print_custom "CoreDNS logging/debug configuration detected"
                if should_report "low"; then
                    add_analysis_result "k8s_resources" "coredns-configmap" "low" \
                        "Custom logging/debug configuration in Corefile" \
                        "Review if logging configuration should be preserved" \
                        "$(echo "$corefile" | grep -E "log|debug")"
                fi
            fi
        fi
    fi
}

# Function to analyze Kube-proxy configuration
analyze_kube_proxy() {
    print_info "=== Analyzing Kube-proxy Configuration ==="
    
    local kube_proxy_ds="$CONFIG_DIR/addon-k8s-resources/kube-proxy-daemonset.yaml"
    local kube_proxy_cm="$CONFIG_DIR/addon-k8s-resources/kube-proxy-configmap.yaml"
    local kube_proxy_addon="$CONFIG_DIR/addons/kube-proxy.json"
    
    # Check EKS managed add-on configuration
    if [[ -f "$kube_proxy_addon" ]]; then
        local custom_config=$(jq -r '.addon.configurationValues // empty' "$kube_proxy_addon" 2>/dev/null)
        if [[ -n "$custom_config" && "$custom_config" != "null" ]]; then
            print_custom "Kube-proxy has custom EKS managed configuration"
            add_analysis_result "eks_addons" "kube-proxy" "high" \
                "Custom EKS managed add-on configuration detected" \
                "Preserve using: aws eks update-addon --configuration-values '$custom_config'" \
                "$custom_config"
        fi
    fi
    
    # Check DaemonSet configuration
    if [[ -f "$kube_proxy_ds" ]] && command -v yq >/dev/null 2>&1; then
        print_info "Analyzing Kube-proxy DaemonSet configuration..."
        
        # Check resource settings
        local cpu_request=$(yq eval '.spec.template.spec.containers[0].resources.requests.cpu // "100m"' "$kube_proxy_ds" 2>/dev/null)
        local memory_request=$(yq eval '.spec.template.spec.containers[0].resources.requests.memory // "64Mi"' "$kube_proxy_ds" 2>/dev/null)
        
        if [[ "$cpu_request" != "100m" || "$memory_request" != "64Mi" ]]; then
            print_custom "Kube-proxy custom resource settings"
            if should_report "medium"; then
                add_analysis_result "k8s_resources" "kube-proxy-daemonset" "medium" \
                    "Custom resource settings: cpu request=$cpu_request, memory request=$memory_request" \
                    "Preserve resource settings in DaemonSet configuration" \
                    "resources: {requests: {cpu: $cpu_request, memory: $memory_request}}"
            fi
        fi
    fi
    
    # Check ConfigMap
    if [[ -f "$kube_proxy_cm" ]] && command -v yq >/dev/null 2>&1; then
        print_info "Analyzing Kube-proxy ConfigMap..."
        
        local config_data=$(yq eval '.data."config.conf" // empty' "$kube_proxy_cm" 2>/dev/null)
        if [[ -n "$config_data" ]]; then
            # Check for common customizations
            if echo "$config_data" | grep -q "mode:.*ipvs"; then
                print_custom "Kube-proxy using IPVS mode"
                if should_report "high"; then
                    add_analysis_result "k8s_resources" "kube-proxy-configmap" "high" \
                        "Custom IPVS mode configuration" \
                        "Preserve IPVS mode setting in ConfigMap" \
                        "mode: ipvs"
                fi
            fi
            
            if echo "$config_data" | grep -q "clusterCIDR:"; then
                local cluster_cidr=$(echo "$config_data" | grep "clusterCIDR:" | awk '{print $2}')
                print_custom "Kube-proxy custom cluster CIDR: $cluster_cidr"
                if should_report "high"; then
                    add_analysis_result "k8s_resources" "kube-proxy-configmap" "high" \
                        "Custom cluster CIDR: $cluster_cidr" \
                        "Preserve cluster CIDR setting in ConfigMap" \
                        "clusterCIDR: $cluster_cidr"
                fi
            fi
        fi
    fi
}

# Function to analyze CSI drivers
analyze_csi_drivers() {
    if [[ -n "$SINGLE_ADDON" ]]; then
        print_info "=== Analyzing $SINGLE_ADDON Configuration ==="
    else
        print_info "=== Analyzing CSI Drivers Configuration ==="
    fi
    
    # EBS CSI Driver - only analyze if not in single addon mode or if it's the target addon
    if [[ -z "$SINGLE_ADDON" || "$SINGLE_ADDON" == "aws-ebs-csi-driver" ]]; then
        local ebs_controller="$CONFIG_DIR/addon-k8s-resources/ebs-csi-controller-deployment.yaml"
        local ebs_node="$CONFIG_DIR/addon-k8s-resources/ebs-csi-node-daemonset.yaml"
        local ebs_addon="$CONFIG_DIR/addons/aws-ebs-csi-driver.json"
    
    if [[ -f "$ebs_addon" ]]; then
        local custom_config=$(jq -r '.addon.configurationValues // empty' "$ebs_addon" 2>/dev/null)
        if [[ -n "$custom_config" && "$custom_config" != "null" ]]; then
            print_custom "EBS CSI Driver has custom EKS managed configuration"
            add_analysis_result "eks_addons" "aws-ebs-csi-driver" "high" \
                "Custom EKS managed add-on configuration detected" \
                "Preserve using: aws eks update-addon --configuration-values '$custom_config'" \
                "$custom_config"
        fi
    fi
    
    # Check for custom resource settings in CSI components
    for csi_file in "$ebs_controller" "$ebs_node"; do
        if [[ -f "$csi_file" ]] && command -v yq >/dev/null 2>&1; then
            local component=$(basename "$csi_file" .yaml)
            
            # Check for custom resource limits
            local has_limits=$(yq eval '.spec.template.spec.containers[].resources.limits // empty' "$csi_file" 2>/dev/null)
            if [[ -n "$has_limits" && "$has_limits" != "null" ]]; then
                print_custom "$component has custom resource limits"
                if should_report "medium"; then
                    add_analysis_result "k8s_resources" "$component" "medium" \
                        "Custom resource limits detected" \
                        "Preserve resource limits in $component configuration" \
                        "$has_limits"
                fi
            fi
            
            # Check for custom environment variables
            local custom_env=$(yq eval '.spec.template.spec.containers[].env[]? | select(.name | test("^CSI_|^AWS_")) | .name + "=" + .value' "$csi_file" 2>/dev/null)
            if [[ -n "$custom_env" ]]; then
                print_custom "$component has custom environment variables"
                if should_report "medium"; then
                    add_analysis_result "k8s_resources" "$component" "medium" \
                        "Custom environment variables detected" \
                        "Preserve environment variables in $component configuration" \
                        "$custom_env"
                fi
            fi
        fi
    done
    fi
    
    # EFS CSI Driver - only analyze if not in single addon mode or if it's the target addon
    if [[ -z "$SINGLE_ADDON" || "$SINGLE_ADDON" == "aws-efs-csi-driver" ]]; then
        local efs_controller="$CONFIG_DIR/addon-k8s-resources/efs-csi-controller-deployment.yaml"
        local efs_node="$CONFIG_DIR/addon-k8s-resources/efs-csi-node-daemonset.yaml"
        local efs_addon="$CONFIG_DIR/addons/aws-efs-csi-driver.json"
        
        if [[ -f "$efs_addon" ]]; then
            local custom_config=$(jq -r '.addon.configurationValues // empty' "$efs_addon" 2>/dev/null)
            if [[ -n "$custom_config" && "$custom_config" != "null" ]]; then
                print_custom "EFS CSI Driver has custom EKS managed configuration"
                add_analysis_result "eks_addons" "aws-efs-csi-driver" "high" \
                    "Custom EKS managed add-on configuration detected" \
                    "Preserve using: aws eks update-addon --configuration-values '$custom_config'" \
                    "$custom_config"
            fi
        fi
        
        # Check for custom resource settings in EFS CSI components
        for csi_file in "$efs_controller" "$efs_node"; do
            if [[ -f "$csi_file" ]] && command -v yq >/dev/null 2>&1; then
                local component=$(basename "$csi_file" .yaml)
                
                # Check for custom resource limits
                local has_limits=$(yq eval '.spec.template.spec.containers[].resources.limits // empty' "$csi_file" 2>/dev/null)
                if [[ -n "$has_limits" && "$has_limits" != "null" ]]; then
                    print_custom "$component has custom resource limits"
                    if should_report "medium"; then
                        add_analysis_result "k8s_resources" "$component" "medium" \
                            "Custom resource limits detected" \
                            "Preserve resource limits in $component configuration" \
                            "$has_limits"
                    fi
                fi
            fi
        done
    fi
    
    # S3 Mountpoint CSI Driver - only analyze if not in single addon mode or if it's the target addon
    if [[ -z "$SINGLE_ADDON" || "$SINGLE_ADDON" == "aws-mountpoint-s3-csi-driver" ]]; then
        local s3_controller="$CONFIG_DIR/addon-k8s-resources/s3-csi-controller-deployment.yaml"
        local s3_node="$CONFIG_DIR/addon-k8s-resources/s3-csi-node-daemonset.yaml"
        local s3_addon="$CONFIG_DIR/addons/aws-mountpoint-s3-csi-driver.json"
        
        if [[ -f "$s3_addon" ]]; then
            local custom_config=$(jq -r '.addon.configurationValues // empty' "$s3_addon" 2>/dev/null)
            if [[ -n "$custom_config" && "$custom_config" != "null" ]]; then
                print_custom "S3 Mountpoint CSI Driver has custom EKS managed configuration"
                add_analysis_result "eks_addons" "aws-mountpoint-s3-csi-driver" "high" \
                    "Custom EKS managed add-on configuration detected" \
                    "Preserve using: aws eks update-addon --configuration-values '$custom_config'" \
                    "$custom_config"
            fi
        fi
        
        # Check for custom resource settings in S3 CSI components
        for csi_file in "$s3_controller" "$s3_node"; do
            if [[ -f "$csi_file" ]] && command -v yq >/dev/null 2>&1; then
                local component=$(basename "$csi_file" .yaml)
                
                # Check for custom resource limits
                local has_limits=$(yq eval '.spec.template.spec.containers[].resources.limits // empty' "$csi_file" 2>/dev/null)
                if [[ -n "$has_limits" && "$has_limits" != "null" ]]; then
                    print_custom "$component has custom resource limits"
                    if should_report "medium"; then
                        add_analysis_result "k8s_resources" "$component" "medium" \
                            "Custom resource limits detected" \
                            "Preserve resource limits in $component configuration" \
                            "$has_limits"
                    fi
                fi
            fi
        done
    fi
}

# Function to analyze IAM configurations
analyze_iam() {
    print_info "=== Analyzing IAM Configurations ==="
    
    if [[ -d "$CONFIG_DIR/iam" ]]; then
        # Check for custom IAM policies
        for iam_file in "$CONFIG_DIR/iam"/role-*-inline-policies.json; do
            if [[ -f "$iam_file" ]]; then
                local role_name=$(basename "$iam_file" | sed 's/role-\(.*\)-inline-policies.json/\1/')
                local inline_policies=$(jq -r '.PolicyNames[]? // empty' "$iam_file" 2>/dev/null)
                
                if [[ -n "$inline_policies" ]]; then
                    print_custom "IAM role $role_name has custom inline policies"
                    if should_report "high"; then
                        add_analysis_result "iam" "$role_name" "high" \
                            "Custom inline IAM policies detected: $inline_policies" \
                            "Document and preserve custom IAM policies during role recreation" \
                            "$inline_policies"
                    fi
                fi
            fi
        done
        
        # Check for custom attached policies (non-AWS managed)
        for iam_file in "$CONFIG_DIR/iam"/role-*-attached-policies.json; do
            if [[ -f "$iam_file" ]]; then
                local role_name=$(basename "$iam_file" | sed 's/role-\(.*\)-attached-policies.json/\1/')
                local custom_policies=$(jq -r '.AttachedPolicies[]? | select(.PolicyArn | test("^arn:aws:iam::[0-9]+:policy/") and (test("aws-service-role") | not)) | .PolicyName' "$iam_file" 2>/dev/null)
                
                if [[ -n "$custom_policies" ]]; then
                    print_custom "IAM role $role_name has custom managed policies"
                    if should_report "high"; then
                        add_analysis_result "iam" "$role_name" "high" \
                            "Custom managed IAM policies detected: $custom_policies" \
                            "Ensure custom managed policies are attached during role recreation" \
                            "$custom_policies"
                    fi
                fi
            fi
        done
    fi
}

# Function to analyze security groups
analyze_security_groups() {
    print_info "=== Analyzing Security Group Configurations ==="
    
    if [[ -d "$CONFIG_DIR/security-groups" ]]; then
        for sg_file in "$CONFIG_DIR/security-groups"/*.json; do
            if [[ -f "$sg_file" ]]; then
                local sg_id=$(basename "$sg_file" .json)
                
                # Check for custom rules (non-standard ports or sources)
                local custom_ingress=$(jq -r '.SecurityGroups[0].IpPermissions[]? | select(.FromPort != 22 and .FromPort != 80 and .FromPort != 443 and .FromPort != 6443 and .FromPort != 10250) | "Port: " + (.FromPort|tostring) + "-" + (.ToPort|tostring) + " Protocol: " + .IpProtocol' "$sg_file" 2>/dev/null)
                
                if [[ -n "$custom_ingress" ]]; then
                    print_custom "Security group $sg_id has custom ingress rules"
                    if should_report "medium"; then
                        add_analysis_result "security_groups" "$sg_id" "medium" \
                            "Custom security group ingress rules detected" \
                            "Document and preserve custom security group rules" \
                            "$custom_ingress"
                    fi
                fi
                
                # Check for custom IP ranges (non-VPC CIDR)
                local custom_cidrs=$(jq -r '.SecurityGroups[0].IpPermissions[].IpRanges[]? | select(.CidrIp != "0.0.0.0/0") | .CidrIp' "$sg_file" 2>/dev/null)
                
                if [[ -n "$custom_cidrs" ]]; then
                    print_custom "Security group $sg_id has custom CIDR blocks"
                    if should_report "medium"; then
                        add_analysis_result "security_groups" "$sg_id" "medium" \
                            "Custom CIDR blocks detected: $custom_cidrs" \
                            "Preserve custom CIDR block configurations" \
                            "$custom_cidrs"
                    fi
                fi
            fi
        done
    fi
}

# Function to analyze custom resources
analyze_custom_resources() {
    print_info "=== Analyzing Custom Resources ==="
    
    if [[ -d "$CONFIG_DIR/custom-resources" ]]; then
        for cr_file in "$CONFIG_DIR/custom-resources"/*.yaml; do
            if [[ -f "$cr_file" ]]; then
                local resource_type=$(basename "$cr_file" .yaml)
                
                if command -v yq >/dev/null 2>&1; then
                    local resource_count=$(yq eval '.items | length' "$cr_file" 2>/dev/null)
                    if [[ "$resource_count" -gt 0 ]]; then
                        print_custom "Custom resources found: $resource_type ($resource_count items)"
                        if should_report "medium"; then
                            add_analysis_result "custom_resources" "$resource_type" "medium" \
                                "Custom resources detected: $resource_count items" \
                                "Document and preserve custom resource configurations" \
                                "Count: $resource_count"
                        fi
                    fi
                fi
            fi
        done
    fi
}

# Function to generate persistence recommendations
generate_recommendations() {
    print_info "=== Generating Persistence Recommendations ==="
    
    local recommendations_file="$OUTPUT_DIR/persistence-recommendations.$FORMAT"
    
    case "$FORMAT" in
        "json")
            cp "$RESULTS_FILE" "$recommendations_file"
            ;;
        "yaml")
            generate_yaml_recommendations "$recommendations_file"
            ;;
        *)
            generate_text_recommendations "$recommendations_file"
            ;;
    esac
    
    print_success "Persistence recommendations generated: $recommendations_file"
}

# Function to generate text recommendations
generate_text_recommendations() {
    local output_file="$1"
    
    cat > "$output_file" << EOF
# EKS Persistent Configuration Analysis Report

**Generated:** $(date)
**Configuration Directory:** $CONFIG_DIR
**Severity Filter:** $SEVERITY

## Executive Summary

EOF
    
    # Generate summary statistics
    local total_findings=$(jq '[.analysis[] | length] | add // 0' "$RESULTS_FILE")
    local critical_findings=$(jq '[.analysis[][] | select(.severity == "critical")] | length' "$RESULTS_FILE")
    local high_findings=$(jq '[.analysis[][] | select(.severity == "high")] | length' "$RESULTS_FILE")
    local medium_findings=$(jq '[.analysis[][] | select(.severity == "medium")] | length' "$RESULTS_FILE")
    local low_findings=$(jq '[.analysis[][] | select(.severity == "low")] | length' "$RESULTS_FILE")
    
    cat >> "$output_file" << EOF
- **Total Findings:** $total_findings
- **Critical:** $critical_findings
- **High:** $high_findings  
- **Medium:** $medium_findings
- **Low:** $low_findings

## Persistence Priority

### 🚨 Critical Priority (Must Preserve)
EOF
    
    jq -r '.analysis[][] | select(.severity == "critical") | "- **" + .component + "**: " + .finding + "\n  - Recommendation: " + .recommendation + "\n"' "$RESULTS_FILE" >> "$output_file"
    
    cat >> "$output_file" << EOF

### ⚠️ High Priority (Should Preserve)
EOF
    
    jq -r '.analysis[][] | select(.severity == "high") | "- **" + .component + "**: " + .finding + "\n  - Recommendation: " + .recommendation + "\n"' "$RESULTS_FILE" >> "$output_file"
    
    cat >> "$output_file" << EOF

### 📋 Medium Priority (Review and Preserve if Needed)
EOF
    
    jq -r '.analysis[][] | select(.severity == "medium") | "- **" + .component + "**: " + .finding + "\n  - Recommendation: " + .recommendation + "\n"' "$RESULTS_FILE" >> "$output_file"
    
    if [[ "$SEVERITY" == "low" ]]; then
        cat >> "$output_file" << EOF

### 💡 Low Priority (Optional)
EOF
        jq -r '.analysis[][] | select(.severity == "low") | "- **" + .component + "**: " + .finding + "\n  - Recommendation: " + .recommendation + "\n"' "$RESULTS_FILE" >> "$output_file"
    fi
    
    cat >> "$output_file" << EOF

## Detailed Findings by Category

EOF
    
    # Generate detailed findings by category
    jq -r '.analysis | to_entries[] | "### " + .key + "\n\n" + (.value[] | "#### " + .component + "\n- **Finding:** " + .finding + "\n- **Severity:** " + .severity + "\n- **Recommendation:** " + .recommendation + "\n- **Configuration:** `" + .config_snippet + "`\n") + "\n"' "$RESULTS_FILE" >> "$output_file"
    
    cat >> "$output_file" << EOF

## Pre-Upgrade Checklist

Before performing add-on upgrades, ensure you have:

### 1. EKS Managed Add-on Configurations
EOF
    
    jq -r '.analysis.eks_addons[]? | "- [ ] " + .component + ": " + .recommendation' "$RESULTS_FILE" >> "$output_file"
    
    cat >> "$output_file" << EOF

### 2. Kubernetes Resource Configurations
EOF
    
    jq -r '.analysis.k8s_resources[]? | "- [ ] " + .component + ": " + .recommendation' "$RESULTS_FILE" >> "$output_file"
    
    cat >> "$output_file" << EOF

### 3. IAM Configurations
EOF
    
    jq -r '.analysis.iam[]? | "- [ ] " + .component + ": " + .recommendation' "$RESULTS_FILE" >> "$output_file"
    
    cat >> "$output_file" << EOF

### 4. Security Group Configurations
EOF
    
    jq -r '.analysis.security_groups[]? | "- [ ] " + .component + ": " + .recommendation' "$RESULTS_FILE" >> "$output_file"
    
    cat >> "$output_file" << EOF

## Post-Upgrade Validation

After add-on upgrades, verify:

1. **Configuration Preservation**: Run the comparison script to ensure custom configurations are preserved
2. **Functionality Testing**: Test all custom features and configurations
3. **Performance Monitoring**: Monitor cluster performance for any degradation
4. **Security Validation**: Verify security group rules and IAM permissions

## Configuration Preservation Commands

### Check Current EKS Add-on Configurations
\`\`\`bash
# Review current add-on configurations before upgrade
EOF
    
    jq -r '.analysis.eks_addons[]? | "aws eks describe-addon --cluster-name $CLUSTER_NAME --addon-name " + .component' "$RESULTS_FILE" >> "$output_file"
    
    cat >> "$output_file" << EOF
\`\`\`

### Monitor Configuration Changes
\`\`\`bash
# Compare configurations before and after upgrade
./collect-eks-nodegroup-config.sh -c $CLUSTER_NAME -r $REGION -o ./post-upgrade-config
./compare-eks-addon-config.sh -b ./pre-upgrade-config -a ./post-upgrade-config
\`\`\`

### Verify Kubernetes Resource Configurations
\`\`\`bash
# Check current Kubernetes resource configurations
EOF
    
    jq -r '.analysis.k8s_resources[]? | "kubectl get " + (.component | split("-")[0]) + " " + (.component | split("-")[1]) + " -n kube-system -o yaml"' "$RESULTS_FILE" >> "$output_file"
    
    cat >> "$output_file" << EOF
\`\`\`

---

**Note:** This analysis is based on the collected configuration at the time of analysis. 
Always verify current configurations before making changes.
EOF
}

# Function to generate YAML recommendations
generate_yaml_recommendations() {
    local output_file="$1"
    
    cat > "$output_file" << EOF
# EKS Persistent Configuration Analysis
metadata:
  generated: $(date -Iseconds)
  config_directory: $CONFIG_DIR
  severity_filter: $SEVERITY

summary:
EOF
    
    local total_findings=$(jq '[.analysis[] | length] | add // 0' "$RESULTS_FILE")
    local critical_findings=$(jq '[.analysis[][] | select(.severity == "critical")] | length' "$RESULTS_FILE")
    local high_findings=$(jq '[.analysis[][] | select(.severity == "high")] | length' "$RESULTS_FILE")
    local medium_findings=$(jq '[.analysis[][] | select(.severity == "medium")] | length' "$RESULTS_FILE")
    local low_findings=$(jq '[.analysis[][] | select(.severity == "low")] | length' "$RESULTS_FILE")
    
    cat >> "$output_file" << EOF
  total_findings: $total_findings
  critical: $critical_findings
  high: $high_findings
  medium: $medium_findings
  low: $low_findings

findings:
EOF
    
    # Convert JSON to YAML format
    jq -r '.analysis | to_entries[] | "  " + .key + ":" + "\n" + (.value[] | "    - component: " + .component + "\n" + "      severity: " + .severity + "\n" + "      finding: \"" + .finding + "\"\n" + "      recommendation: \"" + .recommendation + "\"\n" + "      config_snippet: \"" + .config_snippet + "\"\n")' "$RESULTS_FILE" >> "$output_file"
}

# Function to generate analysis summary
generate_analysis_summary() {
    print_info "=== Analysis Summary ==="
    
    local total_findings=$(jq '[.analysis[][] | length] | add // 0' "$RESULTS_FILE")
    local critical_findings=$(jq '[.analysis[][] | select(.severity == "critical") | length] | add // 0' "$RESULTS_FILE")
    local high_findings=$(jq '[.analysis[][] | select(.severity == "high") | length] | add // 0' "$RESULTS_FILE")
    
    echo "Total findings: $total_findings"
    echo "Critical findings: $critical_findings"
    echo "High priority findings: $high_findings"
    
    if [[ $critical_findings -gt 0 ]]; then
        print_warning "Found $critical_findings critical configurations that must be preserved"
    fi
    
    if [[ $high_findings -gt 0 ]]; then
        print_warning "Found $high_findings high priority configurations that should be preserved"
    fi
    
    echo ""
    print_info "Review the generated reports for detailed configuration preservation guidance"
}

# Main execution
main() {
    if [[ -n "$SINGLE_ADDON" ]]; then
        print_info "Starting single addon analysis for: $SINGLE_ADDON"
    else
        print_info "Starting persistent configuration analysis..."
    fi
    
    # Run analysis functions based on single addon mode or full analysis
    if [[ -z "$SINGLE_ADDON" ]]; then
        # Full analysis mode
        analyze_vpc_cni
        analyze_coredns
        analyze_kube_proxy
        analyze_csi_drivers
        analyze_iam
        analyze_security_groups
        analyze_custom_resources
    else
        # Single addon mode
        case "$SINGLE_ADDON" in
            vpc-cni)
                analyze_vpc_cni
                ;;
            coredns)
                analyze_coredns
                ;;
            kube-proxy)
                analyze_kube_proxy
                ;;
            aws-ebs-csi-driver|aws-efs-csi-driver|aws-mountpoint-s3-csi-driver)
                analyze_csi_drivers
                ;;
        esac
    fi
    
    # Generate recommendations
    generate_recommendations
    
    # Generate analysis summary
    generate_analysis_summary
    
    # Summary
    print_success "Persistent configuration analysis completed!"
    print_info "Results saved to: $OUTPUT_DIR"
    
    # Show summary statistics
    local total_findings=$(jq '[.analysis[] | length] | add // 0' "$RESULTS_FILE")
    local critical_findings=$(jq '[.analysis[][] | select(.severity == "critical")] | length' "$RESULTS_FILE")
    local high_findings=$(jq '[.analysis[][] | select(.severity == "high")] | length' "$RESULTS_FILE")
    local medium_findings=$(jq '[.analysis[][] | select(.severity == "medium")] | length' "$RESULTS_FILE")
    
    echo ""
    print_info "=== Analysis Summary ==="
    print_info "Total findings: $total_findings"
    if [[ "$critical_findings" -gt 0 ]]; then
        print_error "Critical configurations: $critical_findings"
    fi
    if [[ "$high_findings" -gt 0 ]]; then
        print_warning "High priority configurations: $high_findings"
    fi
    if [[ "$medium_findings" -gt 0 ]]; then
        print_info "Medium priority configurations: $medium_findings"
    fi
    
    if [[ "$critical_findings" -gt 0 || "$high_findings" -gt 0 ]]; then
        print_warning "⚠ Critical or high-priority custom configurations detected!"
        print_warning "Review the recommendations before performing add-on upgrades."
    else
        print_success "✓ No critical custom configurations detected."
    fi
}

# Check dependencies
check_dependencies() {
    local missing_deps=()
    
    if ! command -v jq >/dev/null 2>&1; then
        missing_deps+=("jq")
    fi
    
    if [[ ${#missing_deps[@]} -gt 0 ]]; then
        print_error "Missing required dependencies: ${missing_deps[*]}"
        print_info "Please install the missing dependencies and try again"
        exit 1
    fi
    
    # Optional dependencies
    if ! command -v yq >/dev/null 2>&1; then
        print_warning "yq not found - YAML analysis will be limited"
    fi
}

# Run dependency check and main function
check_dependencies
main